In [1]:
import plotly.express as px
import pandas as pd
import numpy as np
import seaborn as sns
sns.set()
import matplotlib.pyplot as plt
from folium import plugins
import folium
import geopandas as gpd
from folium.plugins import FastMarkerCluster
from branca.colormap import LinearColormap
import csv
Becasue "listing_details.csv" has 104 columns and thus contains too much noise, we have created a file called "listing_valid.csv" which contains more sensable and organised data for us to analyse¶
In [2]:
df = pd.read_csv('listing_valid.csv')
df.info()
In [3]:
df=df[df['accommodates']==2]
df=df[df['room_type']=='Private room']
df=df[df['property_type']=='Apartment']
df=df[df['price'] < 400]
df.info()
1.1 Encode the superhost as a dummy column and update the data set¶
In [4]:
df = df.merge(pd.get_dummies(df['host_is_superhost']), left_index = True, right_index = True)
df = df.drop(columns = ['f']).rename(columns = {'t':'super host'})
df.info()
In [5]:
df.head()
Out[5]:
In [6]:
geo = gpd.read_file("neighbourhoods.geojson")
mean_price = pd.DataFrame(df.groupby('neighbourhood_group')['price'].mean().sort_values(ascending=True)).reset_index()
In [7]:
geo.head()
Out[7]:
In [8]:
barca = pd.merge(geo,mean_price, on='neighbourhood_group', how="left")
In [9]:
barca.rename(columns={'price': 'average_price'}, inplace=True)
barca.average_price = barca.average_price.round(decimals=0)
barcelona_map = folium.Map(location=[41.38879, 2.15899], zoom_start=12)
accidents = plugins.MarkerCluster().add_to(barcelona_map)
color_scale = LinearColormap(['yellow','red'], vmin = 42, vmax=50)
map_dict = barca.set_index('neighbourhood_group')['average_price'].to_dict()
In [10]:
barca.head()
Out[10]:
In [11]:
features = df.columns.sort_values().tolist()
def get_color(feature):
value = map_dict.get(feature['properties']['neighbourhood_group'])
return color_scale(value)
folium.GeoJson(data=barca,
name='Barcelona',
tooltip=folium.features.GeoJsonTooltip(fields=['neighbourhood_group', 'average_price'],
labels=True,
sticky=False,
),
style_function= lambda feature: {
'fillColor': get_color(feature),
'color': 'black',
'weight': 1,
'dashArray': '5, 5',
'fillOpacity':0.5,
},
highlight_function=lambda feature: {'weight':1, 'fillColor': get_color(feature), 'fillOpacity': 0.5}).add_to(barcelona_map)
Out[11]:
In [12]:
import branca
colormap = branca.colormap.linear.YlOrRd_05.scale(36, 60)
colormap = colormap.to_step('bottomright',index=[36, 43, 49, 55])
colormap.caption = 'Average Airbnb price per night (per neighbourhood)'
colormap.add_to(barcelona_map)
folium.map.LayerControl('topright', collapsed=False).add_to(colormap)
barcelona_map
Out[12]:
In [13]:
from plotly.subplots import make_subplots
import plotly.graph_objects as go
fig = make_subplots(rows=1, cols=5, shared_yaxes=True, vertical_spacing=0.02)
fig.add_trace(go.Scatter(x=df["review_scores_communication"], y=df["price"],mode='markers' ), row=1, col=1)
fig.add_trace(go.Scatter(x=df["review_scores_location"], y=df["price"], mode='markers'), row=1, col=2)
fig.add_trace(go.Scatter(x=df["review_scores_value"], y=df["price"], mode='markers'), row=1, col=3)
fig.add_trace(go.Scatter(x=df["review_scores_cleanliness"], y=df["price"], mode='markers'), row=1, col=4)
fig.add_trace(go.Scatter(x=df["review_scores_accuracy"], y=df["price"], mode='markers'), row=1, col=5)
# Update xaxis properties
fig.update_xaxes(title_text="communication rating", row=1, col=1)
fig.update_xaxes(title_text="location rating", row=1, col=2)
fig.update_xaxes(title_text="value rating", row=1, col=3)
fig.update_xaxes(title_text="cleanliness rating", row=1, col=4)
fig.update_xaxes(title_text="accuracy rating", row=1, col=5)
fig.update_yaxes(title_text="price", row=1, col=1)
fig.update_layout(showlegend=False, title_text="Effect of Review Scores on Price")
fig.show()
4. Price vs A group of factors¶
4.1 Data Filtering¶
In [14]:
t4 = df.filter(items= ['id','neighbourhood','latitude','longitude','review_scores_rating','price','review_scores_checkin','review_scores_communication',\
'review_scores_accuracy','review_scores_location','review_scores_cleanliness','review_scores_value','neighbourhood_group',\
'host_is_superhost','number_of_reviews','super host'])
t4['latitude'] = t4['latitude'].astype(float)
t4.head()
Out[14]:
4.2 Distance Calculating Function¶
We take the center of Barcelona as (41.3851, 2.1734) and compute the distance of each single neighbourhood to the city center, with unit in Kilometers¶
Algorithm Refernce: https://gist.github.com/rochacbruno/2883505
In [15]:
import math
def distance(origin, destination):
lat1, lon1 = origin
lat2, lon2 = destination
radius = 6371 # km
dlat = math.radians(lat2-lat1)
dlon = math.radians(lon2-lon1)
a = math.sin(dlat/2) * math.sin(dlat/2) + math.cos(math.radians(lat1)) \
* math.cos(math.radians(lat2)) * math.sin(dlon/2) * math.sin(dlon/2)
c = 2 * math.atan2(math.sqrt(a), math.sqrt(1-a))
d = radius * c
return d
In [16]:
t6 = t4.groupby('neighbourhood').mean()
for i,r in t6.iterrows():
t6.at[i,'distance'] = distance((r['latitude'], r['longitude']), (41.3851, 2.1734))
temp = t6.merge(t4.filter(items = ['neighbourhood','neighbourhood_group']).set_index('neighbourhood'), left_index = True, right_index = True)
t7 = temp.drop_duplicates()
modify = {"review_scores_rating": "review rating", "review_scores_checkin": "checkin rating",\
"review_scores_communication": "communication rating",\
"review_scores_accuracy":'accuracy rating',\
'review_scores_location':'location rating',\
'review_scores_cleanliness': 'cleanliness rating',\
'review_scores_value': 'value rating',
'number_of_reviews': 'number of reviews',
'neighbourhood_group':'neighbourhood group' }
t7 = t7.rename(columns = modify).reset_index()
t7.head()
Out[16]:
In [17]:
t7.info()
We have 70 neighbourhoods with its average ratings, price, distances, number of reviews etc. Now we plot the correlation table in heatmap¶
In [18]:
correlation_table = t7.filter(items = ['price','checkin rating','communication rating',\
'accuracy rating','location rating','cleanliness rating','value rating','number of reviews','distance','super host']).corr()
correlation_table
Out[18]:
In [19]:
import plotly.figure_factory as ff
z=correlation_table.values.tolist()
z_text = np.around(z, decimals=3)
fig = ff.create_annotated_heatmap(
z=correlation_table.values.tolist(),
x=list(correlation_table.columns),
y=list(correlation_table.index),
colorscale='Viridis',
annotation_text=z_text
)
fig.show()
5. Price vs Distance from the center of Barcelona (41.3851, 2.1734)¶
In [20]:
fig = px.histogram(df, x="number_of_reviews")
fig.show()
In [21]:
df2 = df[df['number_of_reviews']>=4]
In [22]:
t5 = df2.filter(items= ['id','neighbourhood','latitude','longitude','review_scores_rating','price','review_scores_checkin','review_scores_communication',\
'review_scores_accuracy','review_scores_location','review_scores_cleanliness','review_scores_value','neighbourhood_group',\
'host_is_superhost','number_of_reviews','super host'])
t5['latitude'] = t5['latitude'].astype(float)
t5.info()
In [23]:
t6 = t5.groupby('neighbourhood').mean()
for i,r in t6.iterrows():
t6.at[i,'distance'] = distance((r['latitude'], r['longitude']), (41.3851, 2.1734))
t6.info()
As we can see, with review > 4, we get one neighbourhood away, so we have 69 now¶
In [24]:
temp = t6.merge(t5.filter(items = ['neighbourhood','neighbourhood_group']).set_index('neighbourhood'), left_index = True, right_index = True)
t7 = temp.drop_duplicates()
modify = {"review_scores_rating": "review rating", "review_scores_checkin": "checkin rating",\
"review_scores_communication": "communication rating",\
"review_scores_accuracy":'accuracy rating',\
'review_scores_location':'location rating',\
'review_scores_cleanliness': 'cleanliness rating',\
'review_scores_value': 'value rating',
'number_of_reviews': 'number of reviews',
'neighbourhood_group':'neighbourhood group' }
t7 = t7.rename(columns = modify).reset_index()
t7.head()
Out[24]:
5.2 Average Price/Day decreases as we go futher from city center, and so does the location rating¶
In [25]:
t7['size'] = [10]*len(t7)
fig2 = px.scatter(t7, x="distance", y="price", color="location rating", size = 'size',trendline='ols',\
hover_name="neighbourhood", hover_data=["neighbourhood group"],\
color_continuous_scale=px.colors.sequential.Viridis)
fig2.show()
6. Dive in to Ciutat Vella¶
6.1 Rename the columns of our original dataset and focus on Ciutat Vella¶
In [26]:
df_new = df.filter(items= ['id','neighbourhood','latitude','longitude','price','review_scores_rating','review_scores_checkin','review_scores_communication',\
'review_scores_accuracy','review_scores_location','review_scores_cleanliness','review_scores_value','neighbourhood_group','super host','number_of_reviews'])
modify = {"review_scores_rating": "review rating", "review_scores_checkin": "checkin rating",\
"review_scores_communication": "communication rating",\
"review_scores_accuracy":'accuracy rating',\
'review_scores_location':'location rating',\
'review_scores_cleanliness': 'cleanliness rating',\
'review_scores_value': 'value rating',\
'number_of_reviews': 'number of reviews',\
'neighbourhood_group': 'neighbourhood group'}
df_new = df_new.rename(columns = modify)
df_new.info()
In [27]:
cv = df_new.where(df_new['neighbourhood group'] == 'Ciutat Vella').dropna(how = 'all')
cv['latitude'] = cv['latitude'].astype(float)
cv.head()
Out[27]:
In [28]:
cv = cv.dropna()
cv.info()
In [29]:
cv['price'].hist()
print(cv['price'].mean())
6.2 Add distance column¶
In [30]:
for i,r in cv.iterrows():
cv.at[i,'distance'] = distance((r['latitude'], r['longitude']), (41.3851, 2.1734))
cv.head()
Out[30]:
6.3 Correlation Heatmap¶
1. We found that whether the host is a super_host, overall review rating and number of reviews are important factors in Ciutat Vella¶
2. The analysis makes sense that distance and location rating is not really important is this area, because the area is close to the coast, being slightly further from city center does not affect the price much¶
In [31]:
import plotly.figure_factory as ff
z=cv[cv.columns[4:]].corr().values.tolist()
z_text = np.around(z, decimals=3)
fig = ff.create_annotated_heatmap(z=cv[cv.columns[4:]].corr().values.tolist(),\
x=list(cv[cv.columns[4:]].corr().columns),\
y=list(cv[cv.columns[4:]].corr().index), annotation_text=z_text, colorscale='Viridis')
fig.show()
In [32]:
ss = df_new.where(df_new['neighbourhood group'] == 'Sarrià-Sant Gervasi').dropna(how = 'all')
ss['latitude'] = ss['latitude'].astype(float)
for i,r in ss.iterrows():
ss.at[i,'distance'] = distance((r['latitude'], r['longitude']), (41.3851, 2.1734))
ss.head()
Out[32]:
In [33]:
ss = ss.dropna()
ss.info()
In [34]:
ss['price'].hist()
print(ss['price'].mean())
Price in Sarrià-Sant Gervasi is slightly higher on avera¶
7.2 Correlation Heatmap¶
As Sarrià-Sant Gervasi is far from the city, the distance factor is more importnat in this area comparing to in Ciutat Vella¶
As the majority of Sarrià-Sant Gervasi is on the mountain, the location matters more! That is why location rating is much more correlated with price¶
Communication rating and Checkin Rating also are important factors¶
In [35]:
import plotly.figure_factory as ff
z=ss[ss.columns[4:]].corr().values.tolist()
z_text = np.around(z, decimals=3)
fig = ff.create_annotated_heatmap(
z=ss[ss.columns[4:]].corr().values.tolist(),
x=list(ss[ss.columns[4:]].corr().columns),
y=list(ss[ss.columns[4:]].corr().index),annotation_text=z_text, colorscale='Viridis'
)
fig.show()